Transform Weather Sensor Data from Waggle to an Xarray Dataset + Plot with ACT#
Imports#
import sage_data_client
from bokeh.models.formatters import DatetimeTickFormatter
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import xarray as xr
import matplotlib.pyplot as plt
from metpy.plots import USCOUNTIES
import act
import numpy as np
import pandas as pd
import warnings
from bokeh.models import DatetimeTickFormatter
def apply_formatter(plot, element):
plot.handles['xaxis'].formatter = DatetimeTickFormatter(hours='%m/%d/%Y \n %H:%M',
minutes='%m/%d/%Y \n %H:%M',
hourmin='%m/%d/%Y \n %H:%M',
days='%m/%d/%Y \n %H:%M',
months='%m/%d/%Y \n %H:%M')
xr.set_options(keep_attrs=True)
warnings.filterwarnings("ignore")
hv.extension("bokeh")
Query for the available Data#
wxt_df = sage_data_client.query(
start="-3h",
filter={
"sensor": "vaisala-wxt536"
}
)
---------------------------------------------------------------------------
TimeoutError Traceback (most recent call last)
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:1348, in AbstractHTTPHandler.do_open(self, http_class, req, **http_conn_args)
1347 try:
-> 1348 h.request(req.get_method(), req.selector, req.data, headers,
1349 encode_chunked=req.has_header('Transfer-encoding'))
1350 except OSError as err: # timeout error
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:1283, in HTTPConnection.request(self, method, url, body, headers, encode_chunked)
1282 """Send a complete request to the server."""
-> 1283 self._send_request(method, url, body, headers, encode_chunked)
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:1329, in HTTPConnection._send_request(self, method, url, body, headers, encode_chunked)
1328 body = _encode(body, 'body')
-> 1329 self.endheaders(body, encode_chunked=encode_chunked)
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:1278, in HTTPConnection.endheaders(self, message_body, encode_chunked)
1277 raise CannotSendHeader()
-> 1278 self._send_output(message_body, encode_chunked=encode_chunked)
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:1038, in HTTPConnection._send_output(self, message_body, encode_chunked)
1037 del self._buffer[:]
-> 1038 self.send(msg)
1040 if message_body is not None:
1041
1042 # create a consistent interface to message_body
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:976, in HTTPConnection.send(self, data)
975 if self.auto_open:
--> 976 self.connect()
977 else:
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:1448, in HTTPSConnection.connect(self)
1446 "Connect to a host on a given (SSL) port."
-> 1448 super().connect()
1450 if self._tunnel_host:
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/http/client.py:942, in HTTPConnection.connect(self)
941 sys.audit("http.client.connect", self, self.host, self.port)
--> 942 self.sock = self._create_connection(
943 (self.host,self.port), self.timeout, self.source_address)
944 # Might fail in OSs that don't implement TCP_NODELAY
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/socket.py:857, in create_connection(address, timeout, source_address)
856 try:
--> 857 raise err
858 finally:
859 # Break explicitly a reference cycle
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/socket.py:845, in create_connection(address, timeout, source_address)
844 sock.bind(source_address)
--> 845 sock.connect(sa)
846 # Break explicitly a reference cycle
TimeoutError: [Errno 110] Connection timed out
During handling of the above exception, another exception occurred:
URLError Traceback (most recent call last)
Cell In[2], line 1
----> 1 wxt_df = sage_data_client.query(
2 start="-3h",
3 filter={
4 "sensor": "vaisala-wxt536"
5 }
6 )
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/site-packages/sage_data_client/query.py:115, in query(start, end, head, tail, filter, endpoint, bucket, experimental_func, experimental_window)
112 headers = {"Accept-Encoding": "gzip"}
113 req = Request(endpoint, data, headers=headers)
--> 115 with urlopen(req) as f:
116 content_encoding = f.headers.get("Content-Encoding", "")
117 if "gzip" in content_encoding:
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:216, in urlopen(url, data, timeout, cafile, capath, cadefault, context)
214 else:
215 opener = _opener
--> 216 return opener.open(url, data, timeout)
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:519, in OpenerDirector.open(self, fullurl, data, timeout)
516 req = meth(req)
518 sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
--> 519 response = self._open(req, data)
521 # post-process response
522 meth_name = protocol+"_response"
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:536, in OpenerDirector._open(self, req, data)
533 return result
535 protocol = req.type
--> 536 result = self._call_chain(self.handle_open, protocol, protocol +
537 '_open', req)
538 if result:
539 return result
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:496, in OpenerDirector._call_chain(self, chain, kind, meth_name, *args)
494 for handler in handlers:
495 func = getattr(handler, meth_name)
--> 496 result = func(*args)
497 if result is not None:
498 return result
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:1391, in HTTPSHandler.https_open(self, req)
1390 def https_open(self, req):
-> 1391 return self.do_open(http.client.HTTPSConnection, req,
1392 context=self._context, check_hostname=self._check_hostname)
File ~/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/urllib/request.py:1351, in AbstractHTTPHandler.do_open(self, http_class, req, **http_conn_args)
1348 h.request(req.get_method(), req.selector, req.data, headers,
1349 encode_chunked=req.has_header('Transfer-encoding'))
1350 except OSError as err: # timeout error
-> 1351 raise URLError(err)
1352 r = h.getresponse()
1353 except:
URLError: <urlopen error [Errno 110] Connection timed out>
Configure Helper Functions and Renaming Conventions#
The renaming is required due to . notations being problematic when working with both Pandas and Xarray data structures.
variable_rename_dict = {'wxt.env.humidity':'air_humidity',
'wxt.env.pressure':'air_pressure',
'wxt.env.temp':'air_temperature',
'wxt.heater.temp':'heater_temperature',
'wxt.heater.volt':'heater_voltage',
'wxt.rain.accumulation':'rain_accumulation',
'wxt.wind.direction':'wind_direction',
'wxt.wind.speed':'wind_speed',
'sys.gps.lat':'latitude',
'sys.gps.lon':'longitude',
}
def generate_data_array(df, variable, rename_variable_dict=variable_rename_dict):
new_variable_name = rename_variable_dict[variable]
df_variable= df.loc[df.name == variable]
ds = df_variable.to_xarray().rename({'value':new_variable_name,
'timestamp':'time',
'meta.vsn':'node'})
ds[new_variable_name].attrs['units'] = df_variable['meta.units'].values[0]
ds['time'] = pd.to_datetime(ds.time)
ds.attrs['datastream'] = ds.node.values[0]
return ds[[new_variable_name]]
def generate_dataset(df, variables, rename_variable_dict=variable_rename_dict):
reindexed = df.set_index(['meta.vsn', 'timestamp'])
return xr.merge([generate_data_array(reindexed, variable) for variable in variables])
Transform the Data to Xarray#
wxt_variables = wxt_df.name.unique()
wxt_variables
array(['wxt.env.humidity', 'wxt.env.pressure', 'wxt.env.temp',
'wxt.heater.temp', 'wxt.heater.volt', 'wxt.rain.accumulation',
'wxt.wind.direction', 'wxt.wind.speed'], dtype=object)
wxt_ds = generate_dataset(wxt_df, wxt_variables).squeeze()
wxt_ds
<xarray.Dataset>
Dimensions: (time: 134566)
Coordinates:
node <U4 'W057'
* time (time) datetime64[ns] 2023-04-14T17:59:43.475266934 ....
Data variables:
air_humidity (time) float64 35.2 35.2 35.2 35.2 ... nan nan nan nan
air_pressure (time) float64 981.4 981.4 981.4 ... 980.1 980.1 980.1
air_temperature (time) float64 24.9 24.9 24.9 24.9 ... 26.3 26.3 26.3
heater_temperature (time) float64 30.9 30.9 30.9 30.9 ... 30.8 30.8 30.8
heater_voltage (time) float64 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
rain_accumulation (time) float64 0.03 0.03 0.03 0.03 ... 0.03 0.03 0.03
wind_direction (time) float64 181.0 181.0 185.0 ... 163.0 163.0 163.0
wind_speed (time) float64 2.9 2.9 2.8 2.8 2.8 ... 2.7 2.8 2.8 2.8
Attributes:
datastream: W057Resample the data to minute requency#
minute_ds = wxt_ds.resample(time='1T').mean()
Visualize using hvplot#
meteogram_variables = ['air_temperature', 'air_humidity', 'wind_speed', 'wind_direction']
plots = []
for variable in meteogram_variables:
plots.append(wxt_ds[variable].hvplot.line(label='10 Hz Data') *
minute_ds[variable].hvplot.line(label='1 Minute Data'))
hv.Layout(plots).cols(2)